{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 音乐网站用户流失预测 -- 参数调优"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 导入工具包"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/lightgbm/__init__.py:46: UserWarning: Starting from version 2.2.1, the library file in distribution wheels for macOS is built by the Apple Clang (Xcode_8.3.3) compiler.\n",
      "This means that in case of installing LightGBM from PyPI via the ``pip install lightgbm`` command, you don't need to install the gcc compiler anymore.\n",
      "Instead of that, you need to install the OpenMP library, which is required for running LightGBM on the system with the Apple Clang compiler.\n",
      "You can install the OpenMP library by the following command: ``brew install libomp``.\n",
      "  \"You can install the OpenMP library by the following command: ``brew install libomp``.\", UserWarning)\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd \n",
    "import numpy as np\n",
    "\n",
    "import lightgbm as lgbm\n",
    "from lightgbm.sklearn import LGBMClassifier\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "data_path = '../data/'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 读取数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "========== Train ==========\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>msno</th>\n",
       "      <th>song_id</th>\n",
       "      <th>source_system_tab</th>\n",
       "      <th>source_screen_name</th>\n",
       "      <th>source_type</th>\n",
       "      <th>target</th>\n",
       "      <th>song_length</th>\n",
       "      <th>genre_ids</th>\n",
       "      <th>artist_name</th>\n",
       "      <th>composer</th>\n",
       "      <th>...</th>\n",
       "      <th>lyricists_count</th>\n",
       "      <th>composer_count</th>\n",
       "      <th>is_featured</th>\n",
       "      <th>artist_count</th>\n",
       "      <th>artist_composer</th>\n",
       "      <th>artist_composer_lyricist</th>\n",
       "      <th>song_lang_boolean</th>\n",
       "      <th>smaller_song</th>\n",
       "      <th>count_song_played</th>\n",
       "      <th>count_artist_played</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>FGtllVqz18RPiwJj/edr2gV78zirAiY/9SmYvia+kCg=</td>\n",
       "      <td>BBzumQNXUHKdEBOB7mAJuzok+IJA1c2Ryg/yzTF6tik=</td>\n",
       "      <td>explore</td>\n",
       "      <td>Explore</td>\n",
       "      <td>online-playlist</td>\n",
       "      <td>1</td>\n",
       "      <td>206471</td>\n",
       "      <td>359</td>\n",
       "      <td>Bastille</td>\n",
       "      <td>Dan Smith| Mark Crew</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>215</td>\n",
       "      <td>1140</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Xumu+NIjS6QYVxDS4/t3SawvJ7viT9hPKXmf0RtLNx8=</td>\n",
       "      <td>bhp/MpSNoqoxOIB+/l8WPqu6jldth4DIpCm3ayXnJqM=</td>\n",
       "      <td>my library</td>\n",
       "      <td>Local playlist more</td>\n",
       "      <td>local-playlist</td>\n",
       "      <td>1</td>\n",
       "      <td>284584</td>\n",
       "      <td>1259</td>\n",
       "      <td>Various Artists</td>\n",
       "      <td>no_composer</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>303617</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Xumu+NIjS6QYVxDS4/t3SawvJ7viT9hPKXmf0RtLNx8=</td>\n",
       "      <td>JNWfrrC7zNN7BdMpsISKa4Mw+xVJYNnxXh3/Epw7QgY=</td>\n",
       "      <td>my library</td>\n",
       "      <td>Local playlist more</td>\n",
       "      <td>local-playlist</td>\n",
       "      <td>1</td>\n",
       "      <td>225396</td>\n",
       "      <td>1259</td>\n",
       "      <td>Nas</td>\n",
       "      <td>N. Jones、W. Adams、J. Lordan、D. Ingle</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>289</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Xumu+NIjS6QYVxDS4/t3SawvJ7viT9hPKXmf0RtLNx8=</td>\n",
       "      <td>2A87tzfnJTSWqD7gIZHisolhe4DMdzkbd6LzO1KHjNs=</td>\n",
       "      <td>my library</td>\n",
       "      <td>Local playlist more</td>\n",
       "      <td>local-playlist</td>\n",
       "      <td>1</td>\n",
       "      <td>255512</td>\n",
       "      <td>1019</td>\n",
       "      <td>Soundway</td>\n",
       "      <td>Kwadwo Donkoh</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>FGtllVqz18RPiwJj/edr2gV78zirAiY/9SmYvia+kCg=</td>\n",
       "      <td>3qm6XTZ6MOCU11x8FIVbAGH5l5uMkT3/ZalWG1oo2Gc=</td>\n",
       "      <td>explore</td>\n",
       "      <td>Explore</td>\n",
       "      <td>online-playlist</td>\n",
       "      <td>1</td>\n",
       "      <td>187802</td>\n",
       "      <td>1011</td>\n",
       "      <td>Brett Young</td>\n",
       "      <td>Brett Young| Kelly Archer| Justin Ebach</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>412</td>\n",
       "      <td>427</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 35 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           msno  \\\n",
       "0  FGtllVqz18RPiwJj/edr2gV78zirAiY/9SmYvia+kCg=   \n",
       "1  Xumu+NIjS6QYVxDS4/t3SawvJ7viT9hPKXmf0RtLNx8=   \n",
       "2  Xumu+NIjS6QYVxDS4/t3SawvJ7viT9hPKXmf0RtLNx8=   \n",
       "3  Xumu+NIjS6QYVxDS4/t3SawvJ7viT9hPKXmf0RtLNx8=   \n",
       "4  FGtllVqz18RPiwJj/edr2gV78zirAiY/9SmYvia+kCg=   \n",
       "\n",
       "                                        song_id source_system_tab  \\\n",
       "0  BBzumQNXUHKdEBOB7mAJuzok+IJA1c2Ryg/yzTF6tik=           explore   \n",
       "1  bhp/MpSNoqoxOIB+/l8WPqu6jldth4DIpCm3ayXnJqM=        my library   \n",
       "2  JNWfrrC7zNN7BdMpsISKa4Mw+xVJYNnxXh3/Epw7QgY=        my library   \n",
       "3  2A87tzfnJTSWqD7gIZHisolhe4DMdzkbd6LzO1KHjNs=        my library   \n",
       "4  3qm6XTZ6MOCU11x8FIVbAGH5l5uMkT3/ZalWG1oo2Gc=           explore   \n",
       "\n",
       "    source_screen_name      source_type  target  song_length genre_ids  \\\n",
       "0              Explore  online-playlist       1       206471       359   \n",
       "1  Local playlist more   local-playlist       1       284584      1259   \n",
       "2  Local playlist more   local-playlist       1       225396      1259   \n",
       "3  Local playlist more   local-playlist       1       255512      1019   \n",
       "4              Explore  online-playlist       1       187802      1011   \n",
       "\n",
       "       artist_name                                 composer  ...  \\\n",
       "0         Bastille                     Dan Smith| Mark Crew  ...   \n",
       "1  Various Artists                              no_composer  ...   \n",
       "2              Nas     N. Jones、W. Adams、J. Lordan、D. Ingle  ...   \n",
       "3         Soundway                            Kwadwo Donkoh  ...   \n",
       "4      Brett Young  Brett Young| Kelly Archer| Justin Ebach  ...   \n",
       "\n",
       "  lyricists_count  composer_count  is_featured  artist_count artist_composer  \\\n",
       "0               0               2            0             0               0   \n",
       "1               0               0            0             0               0   \n",
       "2               0               1            0             0               0   \n",
       "3               0               1            0             0               0   \n",
       "4               0               3            0             0               0   \n",
       "\n",
       "   artist_composer_lyricist  song_lang_boolean  smaller_song  \\\n",
       "0                         0                  0             1   \n",
       "1                         0                  0             0   \n",
       "2                         0                  0             1   \n",
       "3                         0                  0             0   \n",
       "4                         0                  0             1   \n",
       "\n",
       "   count_song_played  count_artist_played  \n",
       "0                215                 1140  \n",
       "1                  1               303617  \n",
       "2                  4                  289  \n",
       "3                  1                    1  \n",
       "4                412                  427  \n",
       "\n",
       "[5 rows x 35 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "========== Test ==========\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>msno</th>\n",
       "      <th>song_id</th>\n",
       "      <th>source_system_tab</th>\n",
       "      <th>source_screen_name</th>\n",
       "      <th>source_type</th>\n",
       "      <th>song_length</th>\n",
       "      <th>genre_ids</th>\n",
       "      <th>artist_name</th>\n",
       "      <th>composer</th>\n",
       "      <th>...</th>\n",
       "      <th>lyricists_count</th>\n",
       "      <th>composer_count</th>\n",
       "      <th>is_featured</th>\n",
       "      <th>artist_count</th>\n",
       "      <th>artist_composer</th>\n",
       "      <th>artist_composer_lyricist</th>\n",
       "      <th>song_lang_boolean</th>\n",
       "      <th>smaller_song</th>\n",
       "      <th>count_song_played</th>\n",
       "      <th>count_artist_played</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>V8ruy7SGk7tDm3zA51DPpn6qutt+vmKMBKa21dp54uM=</td>\n",
       "      <td>WmHKgKMlp1lQMecNdNvDMkvIycZYHnFwDT72I5sIssc=</td>\n",
       "      <td>my library</td>\n",
       "      <td>Local playlist more</td>\n",
       "      <td>local-library</td>\n",
       "      <td>224130</td>\n",
       "      <td>458</td>\n",
       "      <td>梁文音 (Rachel Liang)</td>\n",
       "      <td>Qi Zheng Zhang</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>694</td>\n",
       "      <td>13654</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>V8ruy7SGk7tDm3zA51DPpn6qutt+vmKMBKa21dp54uM=</td>\n",
       "      <td>y/rsZ9DC7FwK5F2PK2D5mj+aOBUJAjuu3dZ14NgE0vM=</td>\n",
       "      <td>my library</td>\n",
       "      <td>Local playlist more</td>\n",
       "      <td>local-library</td>\n",
       "      <td>320470</td>\n",
       "      <td>465</td>\n",
       "      <td>林俊傑 (JJ Lin)</td>\n",
       "      <td>林俊傑</td>\n",
       "      <td>...</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>6090</td>\n",
       "      <td>115325</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>/uQAlrAkaczV+nWCd2sPF2ekvXPRipV7q0l+gbLuxjw=</td>\n",
       "      <td>8eZLFOdGVdXBSqoAv5nsLigeH2BvKXzTQYtUM53I0k4=</td>\n",
       "      <td>discover</td>\n",
       "      <td>NaN</td>\n",
       "      <td>song-based-playlist</td>\n",
       "      <td>315899</td>\n",
       "      <td>2022</td>\n",
       "      <td>Yu Takahashi (高橋優)</td>\n",
       "      <td>Yu Takahashi</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>989</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>1a6oo/iXKatxQx4eS9zTVD+KlSVaAFbTIqVvwLC1Y0k=</td>\n",
       "      <td>ztCf8thYsS4YN3GcIL/bvoxLm/T5mYBVKOO4C9NiVfQ=</td>\n",
       "      <td>radio</td>\n",
       "      <td>Radio</td>\n",
       "      <td>radio</td>\n",
       "      <td>285210</td>\n",
       "      <td>465</td>\n",
       "      <td>U2</td>\n",
       "      <td>The Edge| Adam Clayton| Larry Mullen| Jr.</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>31</td>\n",
       "      <td>698</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>1a6oo/iXKatxQx4eS9zTVD+KlSVaAFbTIqVvwLC1Y0k=</td>\n",
       "      <td>MKVMpslKcQhMaFEgcEQhEfi5+RZhMYlU3eRDpySrH8Y=</td>\n",
       "      <td>radio</td>\n",
       "      <td>Radio</td>\n",
       "      <td>radio</td>\n",
       "      <td>197590</td>\n",
       "      <td>873</td>\n",
       "      <td>Yoga Mr Sound</td>\n",
       "      <td>Neuromancer</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>180</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 35 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   id                                          msno  \\\n",
       "0   0  V8ruy7SGk7tDm3zA51DPpn6qutt+vmKMBKa21dp54uM=   \n",
       "1   1  V8ruy7SGk7tDm3zA51DPpn6qutt+vmKMBKa21dp54uM=   \n",
       "2   2  /uQAlrAkaczV+nWCd2sPF2ekvXPRipV7q0l+gbLuxjw=   \n",
       "3   3  1a6oo/iXKatxQx4eS9zTVD+KlSVaAFbTIqVvwLC1Y0k=   \n",
       "4   4  1a6oo/iXKatxQx4eS9zTVD+KlSVaAFbTIqVvwLC1Y0k=   \n",
       "\n",
       "                                        song_id source_system_tab  \\\n",
       "0  WmHKgKMlp1lQMecNdNvDMkvIycZYHnFwDT72I5sIssc=        my library   \n",
       "1  y/rsZ9DC7FwK5F2PK2D5mj+aOBUJAjuu3dZ14NgE0vM=        my library   \n",
       "2  8eZLFOdGVdXBSqoAv5nsLigeH2BvKXzTQYtUM53I0k4=          discover   \n",
       "3  ztCf8thYsS4YN3GcIL/bvoxLm/T5mYBVKOO4C9NiVfQ=             radio   \n",
       "4  MKVMpslKcQhMaFEgcEQhEfi5+RZhMYlU3eRDpySrH8Y=             radio   \n",
       "\n",
       "    source_screen_name          source_type  song_length genre_ids  \\\n",
       "0  Local playlist more        local-library       224130       458   \n",
       "1  Local playlist more        local-library       320470       465   \n",
       "2                  NaN  song-based-playlist       315899      2022   \n",
       "3                Radio                radio       285210       465   \n",
       "4                Radio                radio       197590       873   \n",
       "\n",
       "          artist_name                                   composer  ...  \\\n",
       "0  梁文音 (Rachel Liang)                             Qi Zheng Zhang  ...   \n",
       "1        林俊傑 (JJ Lin)                                        林俊傑  ...   \n",
       "2  Yu Takahashi (高橋優)                               Yu Takahashi  ...   \n",
       "3                  U2  The Edge| Adam Clayton| Larry Mullen| Jr.  ...   \n",
       "4       Yoga Mr Sound                                Neuromancer  ...   \n",
       "\n",
       "  lyricists_count  composer_count  is_featured  artist_count artist_composer  \\\n",
       "0               0               1            0             0               0   \n",
       "1               2               1            0             0               0   \n",
       "2               1               1            0             0               0   \n",
       "3               0               4            0             0               0   \n",
       "4               0               1            0             0               0   \n",
       "\n",
       "   artist_composer_lyricist  song_lang_boolean  smaller_song  \\\n",
       "0                         0                  0             1   \n",
       "1                         0                  0             0   \n",
       "2                         0                  1             0   \n",
       "3                         0                  0             0   \n",
       "4                         0                  0             1   \n",
       "\n",
       "   count_song_played  count_artist_played  \n",
       "0                694                13654  \n",
       "1               6090               115325  \n",
       "2                  5                  989  \n",
       "3                 31                  698  \n",
       "4                  5                  180  \n",
       "\n",
       "[5 rows x 35 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train = pd.read_csv(data_path + 'LGBM_data/fe_train.csv')\n",
    "test = pd.read_csv(data_path + 'LGBM_data/fe_test.csv')\n",
    "\n",
    "print(\"=\"*10,\"Train\",\"=\"*10)\n",
    "train.head()\n",
    "print(\"=\"*10,\"Test\",\"=\"*10)\n",
    "test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(7377418, 35)"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "(2556790, 35)"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.shape\n",
    "test.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 准备数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "for col in train.columns:\n",
    "    if train[col].dtype == object:\n",
    "        train[col] = train[col].astype('category')\n",
    "        test[col] = test[col].astype('category')\n",
    "\n",
    "\n",
    "X_train = train.drop(['target'], axis=1)\n",
    "y_train = train['target'].values\n",
    "\n",
    "X_test = test.drop(['id'], axis=1)\n",
    "ids = test['id'].values\n",
    "\n",
    "# # 训练集\n",
    "# d_train_final = lgbm.Dataset(X_train, y_train)\n",
    "# # 校验集\n",
    "# watchlist_final = lgbm.Dataset(X_train, y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 参数调优\n",
    "\n",
    "LightGBM的主要的超参包括：\n",
    "\n",
    "- 树的数目n_estimators和学习率learning_rate\n",
    "\n",
    "- 树的最大深度max_depth和树的最大叶子节点数目num_leaves（注意：XGBoost只有max_depth，LightGBM采用叶子优先的方式生成树，num_leaves很重要，设置成比 2^max_depth 小）\n",
    "\n",
    "- 叶子结点的最小样本数:min_data_in_leaf(min_data, min_child_samples)\n",
    "\n",
    "- 每棵树的列采样比例：feature_fraction/colsample_bytree\n",
    "\n",
    "- 每棵树的行采样比例：bagging_fraction（需同时设置bagging_freq=1）/subsample\n",
    "\n",
    "- 正则化参数lambda_l1(reg_alpha), lambda_l2(reg_lambda)\n",
    "\n",
    "- 两个非模型复杂度参数，但会影响模型速度和精度。可根据特征取值范围和样本数目修改这两个参数\n",
    "\n",
    "1）特征的最大bin数目max_bin：默认255；\n",
    "\n",
    "2）用来建立直方图的样本数目subsample_for_bin：默认200000。\n",
    "\n",
    "对n_estimators，用LightGBM内嵌的cv函数调优，因为同XGBoost一样，LightGBM学习的过程内嵌了cv，速度极快。\n",
    "\n",
    "其他参数用GridSearchCV"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Step1. 学习率和学习器数目"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 设置最大迭代次数\n",
    "MAX_ROUNDS = 10000"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "参数附加说明：\n",
    "\n",
    "**train：**\n",
    "\n",
    "- verbose_eval：迭代多少次打印\n",
    "- early_stopping_rounds：有多少次分数没有提高则停止\n",
    "- feval：自定义评价函数\n",
    "- evals_result：评价结果，如果early_stopping_rounds被明确指出的话\n",
    "- importance_type:如果是split那么他返回的结果就是被使用过的特征，如果是gain那么返回的结果就是全部的特征\n",
    "\n",
    "**params：**\n",
    "\n",
    "- bagging_fraction和bagging_freq同时使用可以更快的出结果\n",
    "- lightgbm.cv: 当用回归的时候应该把stratified设为false\n",
    "\n",
    "**ps**\n",
    "\n",
    "lightgbm做gridsearch的时候不用加estimator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[100]\tcv_agg's auc: 0.759985 + 0.000429233\n",
      "[200]\tcv_agg's auc: 0.771623 + 0.000487465\n",
      "[300]\tcv_agg's auc: 0.778697 + 0.000429159\n",
      "[400]\tcv_agg's auc: 0.784107 + 0.000423471\n",
      "[500]\tcv_agg's auc: 0.788205 + 0.000361693\n",
      "[600]\tcv_agg's auc: 0.791299 + 0.000365349\n",
      "[700]\tcv_agg's auc: 0.794147 + 0.000432277\n",
      "[800]\tcv_agg's auc: 0.796426 + 0.000433185\n",
      "[900]\tcv_agg's auc: 0.798211 + 0.000368571\n",
      "[1000]\tcv_agg's auc: 0.80013 + 0.000372279\n",
      "[1100]\tcv_agg's auc: 0.802074 + 0.000484375\n",
      "[1200]\tcv_agg's auc: 0.803653 + 0.000504206\n",
      "[1300]\tcv_agg's auc: 0.804985 + 0.000525066\n",
      "[1400]\tcv_agg's auc: 0.806161 + 0.000503735\n",
      "[1500]\tcv_agg's auc: 0.807287 + 0.000470814\n",
      "[1600]\tcv_agg's auc: 0.808142 + 0.000463082\n",
      "[1700]\tcv_agg's auc: 0.809132 + 0.000405197\n",
      "[1800]\tcv_agg's auc: 0.809937 + 0.000389585\n",
      "[1900]\tcv_agg's auc: 0.81067 + 0.000434271\n",
      "[2000]\tcv_agg's auc: 0.811258 + 0.000448914\n",
      "[2100]\tcv_agg's auc: 0.811823 + 0.00045443\n",
      "[2200]\tcv_agg's auc: 0.812327 + 0.000435449\n",
      "[2300]\tcv_agg's auc: 0.812786 + 0.000443748\n",
      "[2400]\tcv_agg's auc: 0.813169 + 0.000444536\n",
      "[2500]\tcv_agg's auc: 0.81357 + 0.000453593\n",
      "[2600]\tcv_agg's auc: 0.813959 + 0.000471513\n",
      "[2700]\tcv_agg's auc: 0.814425 + 0.000516884\n",
      "[2800]\tcv_agg's auc: 0.814813 + 0.000533194\n",
      "[2900]\tcv_agg's auc: 0.815167 + 0.000551885\n",
      "[3000]\tcv_agg's auc: 0.815456 + 0.000578133\n",
      "[3100]\tcv_agg's auc: 0.815736 + 0.000584668\n",
      "[3200]\tcv_agg's auc: 0.816016 + 0.000581306\n",
      "[3300]\tcv_agg's auc: 0.816267 + 0.000596415\n",
      "[3400]\tcv_agg's auc: 0.816503 + 0.00059172\n",
      "[3500]\tcv_agg's auc: 0.816746 + 0.000596338\n",
      "[3600]\tcv_agg's auc: 0.816972 + 0.000572595\n",
      "[3700]\tcv_agg's auc: 0.817184 + 0.00056508\n",
      "[3800]\tcv_agg's auc: 0.817381 + 0.000562094\n",
      "[3900]\tcv_agg's auc: 0.817564 + 0.000560195\n",
      "[4000]\tcv_agg's auc: 0.817736 + 0.000566561\n",
      "[4100]\tcv_agg's auc: 0.81793 + 0.000564481\n",
      "[4200]\tcv_agg's auc: 0.818105 + 0.000556315\n",
      "[4300]\tcv_agg's auc: 0.818276 + 0.000542843\n",
      "[4400]\tcv_agg's auc: 0.818444 + 0.000547827\n",
      "[4500]\tcv_agg's auc: 0.81859 + 0.00053928\n",
      "[4600]\tcv_agg's auc: 0.818754 + 0.000536726\n",
      "[4700]\tcv_agg's auc: 0.818906 + 0.000535652\n",
      "[4800]\tcv_agg's auc: 0.819044 + 0.000535011\n",
      "[4900]\tcv_agg's auc: 0.819182 + 0.000523128\n",
      "[5000]\tcv_agg's auc: 0.819329 + 0.0005174\n",
      "[5100]\tcv_agg's auc: 0.819468 + 0.000517268\n",
      "[5200]\tcv_agg's auc: 0.819604 + 0.000518259\n",
      "[5300]\tcv_agg's auc: 0.819742 + 0.000517314\n",
      "[5400]\tcv_agg's auc: 0.819871 + 0.000501452\n",
      "[5500]\tcv_agg's auc: 0.819997 + 0.000505582\n",
      "[5600]\tcv_agg's auc: 0.820112 + 0.000514421\n",
      "[5700]\tcv_agg's auc: 0.820217 + 0.000503119\n",
      "[5800]\tcv_agg's auc: 0.82032 + 0.000503827\n",
      "[5900]\tcv_agg's auc: 0.82043 + 0.000504874\n",
      "[6000]\tcv_agg's auc: 0.820535 + 0.000506673\n",
      "[6100]\tcv_agg's auc: 0.820649 + 0.000501179\n",
      "[6200]\tcv_agg's auc: 0.82076 + 0.00050078\n",
      "[6300]\tcv_agg's auc: 0.820885 + 0.000523492\n",
      "[6400]\tcv_agg's auc: 0.820975 + 0.000516675\n",
      "[6500]\tcv_agg's auc: 0.82108 + 0.00051269\n",
      "[6600]\tcv_agg's auc: 0.821167 + 0.000511006\n",
      "[6700]\tcv_agg's auc: 0.821257 + 0.000506139\n",
      "[6800]\tcv_agg's auc: 0.821351 + 0.000494925\n",
      "[6900]\tcv_agg's auc: 0.821442 + 0.000493296\n",
      "[7000]\tcv_agg's auc: 0.821522 + 0.000488791\n",
      "[7100]\tcv_agg's auc: 0.821615 + 0.000489047\n",
      "[7200]\tcv_agg's auc: 0.821692 + 0.000491046\n",
      "[7300]\tcv_agg's auc: 0.821771 + 0.000497179\n",
      "[7400]\tcv_agg's auc: 0.821852 + 0.000494196\n",
      "[7500]\tcv_agg's auc: 0.821941 + 0.000490003\n",
      "[7600]\tcv_agg's auc: 0.822015 + 0.000491731\n",
      "[7700]\tcv_agg's auc: 0.82209 + 0.000495065\n",
      "[7800]\tcv_agg's auc: 0.822161 + 0.000493384\n",
      "[7900]\tcv_agg's auc: 0.822235 + 0.000495792\n",
      "[8000]\tcv_agg's auc: 0.82232 + 0.000498096\n",
      "[8100]\tcv_agg's auc: 0.822386 + 0.000501265\n",
      "[8200]\tcv_agg's auc: 0.822451 + 0.000499563\n",
      "[8300]\tcv_agg's auc: 0.822513 + 0.000508347\n",
      "[8400]\tcv_agg's auc: 0.82257 + 0.000508067\n",
      "[8500]\tcv_agg's auc: 0.822627 + 0.000502292\n",
      "[8600]\tcv_agg's auc: 0.822685 + 0.000502763\n",
      "[8700]\tcv_agg's auc: 0.822731 + 0.000498018\n",
      "[8800]\tcv_agg's auc: 0.822792 + 0.000495565\n",
      "[8900]\tcv_agg's auc: 0.82285 + 0.000481704\n",
      "[9000]\tcv_agg's auc: 0.822914 + 0.000494388\n",
      "[9100]\tcv_agg's auc: 0.822979 + 0.000480448\n",
      "[9200]\tcv_agg's auc: 0.823041 + 0.000470382\n",
      "[9300]\tcv_agg's auc: 0.8231 + 0.000478852\n",
      "[9400]\tcv_agg's auc: 0.823144 + 0.000471838\n",
      "[9500]\tcv_agg's auc: 0.823198 + 0.000478554\n",
      "[9600]\tcv_agg's auc: 0.823248 + 0.00047645\n",
      "[9700]\tcv_agg's auc: 0.823304 + 0.000470611\n",
      "[9800]\tcv_agg's auc: 0.823359 + 0.000471758\n",
      "[9900]\tcv_agg's auc: 0.823399 + 0.000466717\n",
      "[10000]\tcv_agg's auc: 0.823441 + 0.000466764\n",
      "CPU times: user 2d 4h 14min 16s, sys: 25min, total: 2d 4h 39min 17s\n",
      "Wall time: 6h 39min 39s\n"
     ]
    }
   ],
   "source": [
    "# 直接调用lightgbm内嵌的交叉验证(cv)，可对连续的n_estimators参数进行快速交叉验证\n",
    "# 而GridSearchCV只能对有限个参数进行交叉验证，且速度相对较慢\n",
    "params = {\n",
    "    'boosting': 'gbdt',\n",
    "    'objective': 'binary',\n",
    "    'learning_rate': 0.1, #\n",
    "    'num_leaves': 100, #\n",
    "    'max_depth': 10, #\n",
    "    'bagging_fraction': 0.95, \n",
    "    'bagging_freq': 1,\n",
    "    'bagging_seed': 1,\n",
    "    'max_bin': 128,\n",
    "    'feature_fraction_seed': 1,\n",
    "    'feature_fraction': 0.9\n",
    "}\n",
    "\n",
    "data_train = lgbm.Dataset(X_train, y_train, silent=True)\n",
    "\n",
    "# 用了%time后面的代码不能换行\n",
    "%time cv_results = lgbm.cv(params, data_train, num_boost_round=MAX_ROUNDS, nfold=5, stratified=False, shuffle=True, metrics='auc',early_stopping_rounds=50, verbose_eval=100, show_stdv=True, seed=0)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best n_estimators: 10000\n",
      "best cv score: 0.8234411451744881\n"
     ]
    }
   ],
   "source": [
    "print('best n_estimators:', len(cv_results['auc-mean']))\n",
    "print('best cv score:', cv_results['auc-mean'][-1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Step2. max_depth 和 num_leaves"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 18.7 s, sys: 29.2 s, total: 47.9 s\n",
      "Wall time: 12h 37min 52s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=3, error_score='raise-deprecating',\n",
       "       estimator=LGBMClassifier(bagging_fraction=0.95, bagging_freq=1, bagging_seed=1,\n",
       "        boosting='gbdt', boosting_type='gbdt', class_weight=None,\n",
       "        colsample_bytree=1.0, feature_fraction=0.9,\n",
       "        feature_fraction_seed=1, importance_type='split',\n",
       "        learning_rate=0.1, max_bin=128, max_....0, reg_lambda=0.0, silent=False, subsample=1.0,\n",
       "        subsample_for_bin=200000, subsample_freq=0),\n",
       "       fit_params=None, iid='warn', n_jobs=4,\n",
       "       param_grid={'max_depth': [8, 9, 10], 'num_leaves': range(50, 90, 10)},\n",
       "       pre_dispatch='2*n_jobs', refit=False, return_train_score='warn',\n",
       "       scoring='neg_log_loss', verbose=0)"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.6916304091644304\n",
      "{'max_depth': 8, 'num_leaves': 50}\n"
     ]
    }
   ],
   "source": [
    "n_estimators = len(cv_results['auc-mean'])\n",
    "params = {\n",
    "    'boosting': 'gbdt',\n",
    "    'objective': 'binary',\n",
    "#     'n_estimators': n_estimators, # 10000个学习器太慢了！\n",
    "    'n_estimators': 1000,\n",
    "    'learning_rate': 0.1, #\n",
    "    #'num_leaves': 100, #\n",
    "    #'max_depth': 10, #\n",
    "    'bagging_fraction': 0.95, \n",
    "    'bagging_freq': 1,\n",
    "    'bagging_seed': 1,\n",
    "    'max_bin': 128,\n",
    "    'feature_fraction_seed': 1,\n",
    "    'feature_fraction': 0.9\n",
    "}\n",
    "\n",
    "lg = LGBMClassifier(silent=False, **params)\n",
    "\n",
    "max_depth_s = [8, 9, 10]  \n",
    "num_leaves_s = range(50, 90, 10)  # 60,70,80,90\n",
    "tuned_parameters = dict(max_depth=max_depth_s, num_leaves=num_leaves_s)\n",
    "\n",
    "grid_search = GridSearchCV(lg,n_jobs=4,param_grid=tuned_parameters,cv=3,scoring=\"neg_log_loss\",verbose=0,refit=False)\n",
    "%time grid_search.fit(X_train, y_train)\n",
    "\n",
    "print(-grid_search.best_score_)\n",
    "print(grid_search.best_params_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step3: min_data_in_leaf 和 min_sum_hessian_in_leaf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 16 s, sys: 29 s, total: 45 s\n",
      "Wall time: 7h 33min 21s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=3, error_score='raise-deprecating',\n",
       "       estimator=LGBMClassifier(bagging_fraction=0.8, bagging_freq=1, bagging_seed=1,\n",
       "        boosting='gbdt', boosting_type='gbdt', class_weight=None,\n",
       "        colsample_bytree=1.0, feature_fraction=0.8,\n",
       "        feature_fraction_seed=1, importance_type='split',\n",
       "        learning_rate=0.1, max_bin=128, max_d...bda=0.0, silent=False, subsample=1.0,\n",
       "        subsample_for_bin=200000, subsample_freq=0, verbose=0),\n",
       "       fit_params=None, iid='warn', n_jobs=-1,\n",
       "       param_grid={'min_data_in_leaf': range(20, 100, 10), 'min_sum_hessian_in_leaf': [0.001]},\n",
       "       pre_dispatch='2*n_jobs', refit=False, return_train_score='warn',\n",
       "       scoring='neg_log_loss', verbose=0)"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-0.6896628851861047\n",
      "{'min_data_in_leaf': 20, 'min_sum_hessian_in_leaf': 0.001}\n"
     ]
    }
   ],
   "source": [
    "params = {\n",
    "    'boosting': 'gbdt',\n",
    "    'objective': 'binary',\n",
    "    'n_estimators': 1000,\n",
    "    'learning_rate': 0.1, #\n",
    "    \n",
    "    'num_leaves': 50, #\n",
    "    'max_depth': 8, #\n",
    "#     'min_data_in_leaf': int(2 ** (np.random.rand()*3.5 + 9)),\n",
    "#     'min_sum_hessian_in_leaf': 0.1,\n",
    "    \n",
    "    'bagging_fraction': 0.8, \n",
    "    'bagging_freq': 1,\n",
    "    'bagging_seed': 1,\n",
    "    \n",
    "    'feature_fraction_seed': 1,\n",
    "    'feature_fraction': 0.8,\n",
    "    \n",
    "    'max_bin': 128,\n",
    "    \n",
    "#     'num_threads': 8,\n",
    "    'verbose': 0,\n",
    "}\n",
    "\n",
    "lg = LGBMClassifier(silent=False, **params)\n",
    "\n",
    "min_data_in_leafs = range(20, 100, 10)\n",
    "min_sum_hessian_in_leafs = [0.001]\n",
    "tuned_parameters = dict(min_data_in_leaf=min_data_in_leafs, min_sum_hessian_in_leaf = min_sum_hessian_in_leafs)\n",
    "\n",
    "grid_search = GridSearchCV(lg,n_jobs=-1,param_grid=tuned_parameters,cv=3,scoring=\"neg_log_loss\",refit=False)\n",
    "%time grid_search.fit(X_train, y_train)\n",
    "\n",
    "print(grid_search.best_score_)\n",
    "print(grid_search.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 5.4 s, sys: 8.03 s, total: 13.4 s\n",
      "Wall time: 2h 30min 5s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=3, error_score='raise-deprecating',\n",
       "       estimator=LGBMClassifier(bagging_fraction=0.6, bagging_freq=1, bagging_seed=1,\n",
       "        boosting='gbdt', boosting_type='gbdt', class_weight=None,\n",
       "        colsample_bytree=1.0, feature_fraction=0.6,\n",
       "        feature_fraction_seed=1, importance_type='split',\n",
       "        learning_rate=0.1, max_bin=128, max_d...bda=0.0, silent=False, subsample=1.0,\n",
       "        subsample_for_bin=200000, subsample_freq=0, verbose=0),\n",
       "       fit_params=None, iid='warn', n_jobs=4,\n",
       "       param_grid={'min_sum_hessian_in_leaf': [0.001, 0.01, 0.1]},\n",
       "       pre_dispatch='2*n_jobs', refit=False, return_train_score='warn',\n",
       "       scoring='neg_log_loss', verbose=0)"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.6916377402459345\n",
      "{'min_sum_hessian_in_leaf': 0.001}\n"
     ]
    }
   ],
   "source": [
    "params = {\n",
    "    'boosting': 'gbdt',\n",
    "    'objective': 'binary',\n",
    "    'n_estimators': 1000,\n",
    "    'learning_rate': 0.1, #\n",
    "    \n",
    "    'num_leaves': 50, #\n",
    "    'max_depth': 8, #\n",
    "    'min_data_in_leaf': 20,\n",
    "#     'min_sum_hessian_in_leaf': 0.001,\n",
    "    \n",
    "    'bagging_fraction': 0.6, \n",
    "    'bagging_freq': 1,\n",
    "    'bagging_seed': 1,\n",
    "    \n",
    "    'feature_fraction_seed': 1,\n",
    "    'feature_fraction': 0.6,\n",
    "    \n",
    "    'max_bin': 128,\n",
    "    \n",
    "    'verbose': 0,\n",
    "}\n",
    "\n",
    "lg = LGBMClassifier(silent=False, **params)\n",
    "\n",
    "min_sum_hessian_in_leafs = [0.001, 0.01, 0.1]\n",
    "tuned_parameters = dict(min_sum_hessian_in_leaf = min_sum_hessian_in_leafs)\n",
    "\n",
    "grid_search = GridSearchCV(lg,n_jobs=4,param_grid=tuned_parameters,cv=3,scoring=\"neg_log_loss\",refit=False)\n",
    "%time grid_search.fit(X_train, y_train)\n",
    "\n",
    "print(-grid_search.best_score_)\n",
    "print(grid_search.best_params_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step4: feature_fraction 和 bagging_fraction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 29.1 s, sys: 52.4 s, total: 1min 21s\n",
      "Wall time: 13h 26min 51s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=3, error_score='raise-deprecating',\n",
       "       estimator=LGBMClassifier(bagging_freq=1, bagging_seed=1, boosting='gbdt',\n",
       "        boosting_type='gbdt', class_weight=None, colsample_bytree=1.0,\n",
       "        feature_fraction_seed=1, importance_type='split',\n",
       "        learning_rate=0.1, max_bin=128, max_depth=10, min_child_samples=20,\n",
       "        min_child_wei... silent=False,\n",
       "        subsample=1.0, subsample_for_bin=200000, subsample_freq=0,\n",
       "        verbose=0),\n",
       "       fit_params=None, iid='warn', n_jobs=4,\n",
       "       param_grid={'bagging_fraction': array([0.6, 0.7, 0.8, 0.9]), 'feature_fraction': array([0.6, 0.7, 0.8, 0.9])},\n",
       "       pre_dispatch='2*n_jobs', refit=False, return_train_score='warn',\n",
       "       scoring='neg_log_loss', verbose=0)"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.6873422536623357\n",
      "{'bagging_fraction': 0.6, 'feature_fraction': 0.8999999999999999}\n"
     ]
    }
   ],
   "source": [
    "params = {\n",
    "    'boosting': 'gbdt',\n",
    "    'objective': 'binary',\n",
    "    'n_estimators': 1000,\n",
    "    'learning_rate': 0.1, #\n",
    "    \n",
    "    'num_leaves': 50, #\n",
    "    'max_depth': 8, #\n",
    "    'min_data_in_leaf': 20,\n",
    "    'min_sum_hessian_in_leaf': 0.001,\n",
    "    \n",
    "#     'bagging_fraction': 0.95, \n",
    "    'bagging_freq': 1,\n",
    "    'bagging_seed': 1,\n",
    "    \n",
    "    'feature_fraction_seed': 1,\n",
    "#     'feature_fraction': 0.9,\n",
    "    \n",
    "    'max_bin': 128,\n",
    "    \n",
    "    'verbose': 0,\n",
    "}\n",
    "\n",
    "lg = LGBMClassifier(silent=False, **params)\n",
    "\n",
    "bagging_fractions = np.arange(0.6, 1, 0.1)\n",
    "feature_fractions = np.arange(0.6, 1, 0.1)\n",
    "tuned_parameters = dict(bagging_fraction=bagging_fractions, feature_fraction=feature_fractions)\n",
    "\n",
    "grid_search = GridSearchCV(lg,n_jobs=4,param_grid=tuned_parameters,cv=3,scoring=\"neg_log_loss\",refit=False)\n",
    "%time grid_search.fit(X_train, y_train)\n",
    "\n",
    "print(-grid_search.best_score_)\n",
    "print(grid_search.best_params_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step5: 正则化参数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 6.97 s, sys: 9.89 s, total: 16.9 s\n",
      "Wall time: 4h 38min 49s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=3, error_score='raise-deprecating',\n",
       "       estimator=LGBMClassifier(bagging_fraction=0.6, bagging_freq=1, bagging_seed=1,\n",
       "        boosting='gbdt', boosting_type='gbdt', class_weight=None,\n",
       "        colsample_bytree=1.0, feature_fraction=0.9,\n",
       "        feature_fraction_seed=1, importance_type='split',\n",
       "        learning_rate=0.1, max_bin=128, max_d...bda=0.0, silent=False, subsample=1.0,\n",
       "        subsample_for_bin=200000, subsample_freq=0, verbose=0),\n",
       "       fit_params=None, iid='warn', n_jobs=4,\n",
       "       param_grid={'reg_alpha': array([1, 2]), 'reg_lambda': array([1, 2])},\n",
       "       pre_dispatch='2*n_jobs', refit=False, return_train_score='warn',\n",
       "       scoring='neg_log_loss', verbose=0)"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.685789855593742\n",
      "{'reg_alpha': 1, 'reg_lambda': 2}\n"
     ]
    }
   ],
   "source": [
    "params = {\n",
    "    'boosting': 'gbdt',\n",
    "    'objective': 'binary',\n",
    "    'n_estimators': 1000,\n",
    "    'learning_rate': 0.1, #\n",
    "    \n",
    "    'num_leaves': 50, #\n",
    "    'max_depth': 8, #\n",
    "    'min_data_in_leaf': 20,\n",
    "    'min_sum_hessian_in_leaf': 0.001,\n",
    "    \n",
    "    'bagging_fraction': 0.6, \n",
    "    'bagging_freq': 1,\n",
    "    'bagging_seed': 1,\n",
    "    \n",
    "    'feature_fraction_seed': 1,\n",
    "    'feature_fraction': 0.9,\n",
    "    \n",
    "#     'reg_alpha': ,\n",
    "#     'reg_lambda': ,\n",
    "    \n",
    "    'max_bin': 128,\n",
    "    'verbose': 0,\n",
    "}\n",
    "\n",
    "lg = LGBMClassifier(silent=False, **params)\n",
    "\n",
    "reg_alphas = np.arange(1, 3, 1)\n",
    "reg_lambdas = np.arange(1, 3, 1)\n",
    "tuned_parameters = dict(reg_alpha=reg_alphas, reg_lambda=reg_lambdas)\n",
    "\n",
    "grid_search = GridSearchCV(lg,n_jobs=4,param_grid=tuned_parameters,cv=3,scoring=\"neg_log_loss\",refit=False)\n",
    "%time grid_search.fit(X_train, y_train)\n",
    "\n",
    "print(-grid_search.best_score_)\n",
    "print(grid_search.best_params_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step6: 降低学习率"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[100]\tcv_agg's auc: 0.728484 + 0.000447058\n",
      "[200]\tcv_agg's auc: 0.736866 + 0.000436995\n",
      "[300]\tcv_agg's auc: 0.743224 + 0.000444632\n",
      "[400]\tcv_agg's auc: 0.747047 + 0.000391933\n",
      "[500]\tcv_agg's auc: 0.749667 + 0.000400793\n",
      "CPU times: user 3h 34min 13s, sys: 16min 22s, total: 3h 50min 36s\n",
      "Wall time: 35min 8s\n"
     ]
    }
   ],
   "source": [
    "params = {\n",
    "    'boosting': 'gbdt',\n",
    "    'objective': 'binary',\n",
    "    'learning_rate': 0.01, #\n",
    "    'num_leaves': 100, #\n",
    "    'max_depth': 10, #\n",
    "    'min_data_in_leaf': 100,#\n",
    "    'min_sum_hessian_in_leaf': 0.01,#\n",
    "    'bagging_fraction': 0.9, #\n",
    "    'bagging_freq': 1,\n",
    "    'bagging_seed': 1,\n",
    "    'max_bin': 255, \n",
    "    'feature_fraction_seed': 1,\n",
    "    'feature_fraction': 0.9, #\n",
    "    'reg_lambda': 2, #\n",
    "    'reg_alpha': 1, #\n",
    "}\n",
    "\n",
    "data_train = lgbm.Dataset(X_train, y_train, silent=True)\n",
    "\n",
    "# 用了%time后面的代码不能换行\n",
    "%time cv_results = lgbm.cv(params, data_train, num_boost_round=500, nfold=5, stratified=False, shuffle=True, metrics='auc',early_stopping_rounds=50, verbose_eval=100, show_stdv=True, seed=0)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "参数调优效果不是很理想，而且运行效率实在太慢了，最后用经验预估的参数对模型进行训练。参考1.2_FE_Train_Test_KKBox_Music"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
